import time

import requests
from pprint import pprint


class JinSeSpider(object):

    def __init__(self):
        self.start_url = "https://api.jinse.com/noah/v1/www/timelines?catelogue_key=www&limit=23&information_id=17344&flag=down"
        self.base_url = "https://api.jinse.com/noah/v1/www/timelines?catelogue_key=www&limit=23&information_id={}&flag=down"
        self.headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.102 Safari/537.36"
        }

    def send_request(self, url):
        response = requests.get(url, headers=self.headers)
        return response.json()

    def parse_data(self, response_data):
        """
        获取到的是数据的字典类型，response.json() 拿到的就是字典类型
        :param response_data: 字典类型的数据
        :return: 数据的列表
        """
        data_list = response_data["data"]["list"]
        content_list = []
        for data in data_list:
            item = {}
            item["title"] = data["title"]
            item["short_title"] = data["short_title"]
            item["type_name"] = data["type_name"]
            item["content"] = data["extra"].get("content") if data["extra"].get("content") else data["extra"].get(
                "summary")
            item["author"] = data["extra"].get("author")
            content_list.append(item)

        # 获取下一页的 information_id
        information_id = response_data.get("data").get("bottom_id")
        return content_list, information_id

    def save_data(self, content_list):
        """
        保存数据的方法
        :param content_list: 要保存的数据的列表
        :return:
        """
        for content in content_list:
            pprint(content)

    def run(self):
        # 1. 先去发送第一页的请求
        response_data = self.send_request(self.start_url)
        # 解析第一页的数据，获取第一页的数据以及下一页的 information_id
        content_list, information_id = self.parse_data(response_data)
        self.save_data(content_list)
        print("下一页的informationid是： ", information_id)

        while True:
            if information_id:
                url = self.base_url.format(information_id)
                response_data = self.send_request(url)
                # 解析第一页的数据，获取第一页的数据以及下一页的 information_id
                content_list, information_id = self.parse_data(response_data)
                self.save_data(content_list)
                time.sleep(3)


if __name__ == '__main__':
    jinse = JinSeSpider()
    jinse.run()
